#!/usr/bin/env perl
# Time-stamp: <09 Nov 04 17:36:58 giron1> 

=head1 NAME

 sam2esps.perl - converts SAM orthographic labels to ESPS ones

=head1 SYNOPSIS

sam2esps.perl [commands] [options] corpus

see also:

L<"COMMANDS">,
L<"OPTIONS">
at the end of this document.

=head1 DESCRIPTION

sam2esps.perl is a perl script to convert orthographic label files 
from SAM format to ESPS label format.

=head1 AUTHOR

 $Id: sam2esps.perl,v 1.1 2007/10/08 09:26:02 schweitz Exp $
 Franck Giron

=head1 LOG

 $Log: sam2esps.perl,v $
 Revision 1.1  2007/10/08 09:26:02  schweitz
 version 2.4, lib vergessen

 Revision /main/10  9-Nov-2004.17:44:43  giron1
 Replaced Iso3 hash key by string which is the standard.
 
 Revision /main/9  17-Jan-2003.12:25:50  giron1
 Updating header to /usr/bin/env perl.
 
 Revision 1.8  2000/10/27 11:24:40  giron1
 Changed Path to string for some options

 Revision 1.7  2000/04/27 15:51:13  giron1
 Pod documentation updated


=cut

#----------------------------------------------------------------------
# perl packages
#----------------------------------------------------------------------

use strict;
use File::Basename;
use File::Path;
use GLOBALS;
use OPTIONS;
use Misc;
use myFind;
use SAM2ESPS;

#----------------------------------------------------------------------

=head1 FUNCTIONS 

Here are the different functions

=head2 wanted

This subroutine is used to match a pattern in the find program

=cut

sub wanted {
    my ($pat,$list) = @_;
    my ($name);

    if (m/^(?:$pat)$/i) {
	$name = $myFind::name;
	push @{$list}, $name;
    }
}

#----------------------------------------------------------------------

=head2 filter

This function convert a single label file

=cut

sub filter{
    my ($opt_l,$com,$fhi,$fho) = @_;

    my ($freq,$out,$i,$endtime);
    my (@values);

    # Skip all lines until sample frequency
    while(<$fhi>) {if (/^SAM:(.*)$/) {$freq = $1;last;}}
    if (!$freq) {die "Error in SAM file, sample frequency is $freq";} 

    # Skip all lines until begin of keys description
    LINE: while(<$fhi>){last LINE if /^LBD/;}

    if ($com ne 'txt' ) {$fho->print("#");}

    # Process all lines

    while(<$fhi>) {
	$out = '';
	if (/^EXT:(.*)$/) {
	    $out = $1;
	    if ($opt_l->{d2u}{Do}) {$out = dos2unix($out);}
#	    $out =~ s/\"//g;
	  SWITCH:{
	      if ($com eq 'word') {
		  $out =~ s/ /_/g; 
	      }
	      if ($com eq 'txt') {
		  $out .= "\n";
	      }
	  }
	}
	if (/^LBR:\s*(.*)$/) {
	    # the fields should be start, end, input gain, minimum level, 
	    # maximum level and orthographic description.
	    @values = split /,\s*/, $1, 6;
#	    $values[5] =~ s/^\s*//;
#	    foreach $i (5 .. $#values) {$out .= $values[$i];}
	    $out = $values[5];
	    if ($opt_l->{d2u}{Do}) {$out = dos2unix($out);}
	    $out =~ s/\s*$//;
	    $endtime = $values[1] / $freq;
	  SWITCH:{
	      if ($com eq 'phon') {
		  $out = "\n$endtime 122 " . $out;
		  last SWITCH;
	      }
	      if ($com eq 'word') {
		  $out =~ s/ /_/g;
		  $out = "\n$endtime 122 " . $out;
		  last SWITCH;
	      }
	      if ($com eq 'txt') {
		  $out .= "\n";
		  last SWITCH;
	      }
	  }
	}
	$fho->print($out);
    }
    $fho->print("\n");
}

#----------------------------------------------------------------------

=head2 ProcessList

This function is the kernel of the conversion of label files

 The following variables have to be defined in the calling routine
    $opt_l    : The list of options coming from the argument list
    $path_db  : The path for the current database
    $ext_l    : The list of file extensions valid for the orthographic label files
                of the current database
    $corp     : The corpus name

=cut

sub ProcessList{
  my ($opt_l,$path_db,$ext_l,$corp) = @_;

  # Test if the list name option has been given
  my $listd = $opt_l->{ldir}{string};
  my $listname = $listd;
  $listname .= $path_db; $listname =~ s/\/$//; $listname .= $opt_l->{elist}{string};

  warn "listd:$listd\n";
  warn "path_db:$path_db\n";
  warn "listname: $listname\n";

  # Open the files list
  if (not -r $listname ) {warn "$listname not existent\n"; return -1;}
  my $fList = new FileHandle "<$listname";
  die "$0: Error by opening $listname\n" unless (defined($fList));
  my @list = map {chop $_; $_ } <$fList>;
  $fList->close;

  warn "list $listname has :",$#list+1," entries\n";

  # test if the corpus belongs to STC standard
  my $stcstandard = grep(/$corp/,@GLOBALS::STC_FORMAT_CORPUS_LIST);

  my $fhin = new FileHandle;
  my $fhout = new FileHandle;

  my $dbdir = $opt_l->{dbdir}{string};

  # Process each entry of the list
  foreach (@list) {
    my $basename = $_;
    my $basein = $basename;

    # Test if the data base belong to STC standard
    if ($stcstandard) {$basein =~ s/(\w+)$/$GLOBALS::UTR_DIR\/$1/;}

    # Suppress the language from basein
    $basein =~ s/$opt_l->{lang}{string}\///;

    # Create directories if necessary
    my $s_root = dirname($basein);
    mkpath($s_root,1,0777);

    # Generate the search pattern from the list of extensions
    my $s_name = basename($basein);
    my $pattern = '';
    foreach (@{$ext_l}) {$pattern .= $s_name . '\.'. $_ .'|';}
    chop $pattern;

    # Find the files with the correct extensions
    my $fileinlist = find(\&wanted,$pattern,$opt_l->{dsound}{string} . $s_root);

    # Process each file of the filein list
    my ($filein,$com);
  LOOP: foreach $filein (@{$fileinlist}) {
      warn "treating $filein:\n";
      my $baseout = $dbdir . $basename;
      foreach $com (keys %{$opt_l}) {
		if ($com ne 'd2u' && $opt_l->{$com}{Do}) {
		  my $fhi = new FileHandle "<$filein";
		  die "$0: Error: by opening $filein\n" unless (defined($fhi));
		  my $fileout = $baseout .$opt_l->{'e' . $com}{string};
		  if (-r $fileout and !-z $fileout) {
			warn "$fileout already DONE.\n";
			$fhi->close;
			next LOOP;
		  }
		  my $fho = new FileHandle "> $fileout";
		  die "$0: Error: by opening $fileout\n" unless (defined($fho));
		  warn "command $com for $fileout\n";
		  filter($opt_l,$com,$fhi,$fho);
		  $fho->close;
		  $fhi->close;
		}
      }
    }
  }
}

#----------------------------------------------------------------------
# NAME
# define_options
#
# DESCRIPTION
# This function defines options which are dependent of the corpus and
# global variables
#
# The following variables have to be defined in the calling routine
#    $corp      : The name of the corpus to be treated
#    $opt_l     : The options list

sub define_options {
  my ($corp,$opt,$popt) = @_;

  # Language option
  $opt->define($popt,'lang',$GLOBALS::DBLANG,
	       "<Iso3>\t: language three letters code from Iso 3166/2T ");
  # Global root
  $opt->define($popt,'groot',$GLOBALS::GLOBALS_PATH,"<dir>\t: global path ");

  # Database directory
  $opt->define($popt,'dbdir',$GLOBALS::DBROOT,"<dir>\t: speech database directory");  

  # Lists directory
  $opt->define($popt,'ldir',$GLOBALS::CORPUS_H->{$corp}{list_dir},
	       "<dir>\t: lists root directory of the defined corpus\n\t\t ");
  # List name
  $opt->define($popt,'lname','',"<name>\t: name of list ");
}

#----------------------------------------------------------------------
# MAIN
#----------------------------------------------------------------------

# Initialisations
my $glob = new GLOBALS;

$glob->{GLOBALS_PATH} = "$ENV{HOME}/db/";
$glob->{DEF_GLOBALS} =  $glob->{GLOBALS_PATH} . 'deu/globals.pl';
$glob->requireglob;

my @corpuslist = keys %{$GLOBALS::CORPUS_H};
my $CORPUS = $corpuslist[0];

# Define the set of options

my $OptL = new OPTIONS;

# Commands
$OptL->lab(Do => 0, Usage => "create labels");
$OptL->word(Do => 0, Usage => "create words");
$OptL->phon(Do => 0, Usage => "create phones");
$OptL->txt(Do => 0, Usage => "create transliteration");
$OptL->d2u(Do => 0, Usage => "convert from dos to unix for the other commands");
$OptL->list(Do => 0, Usage => "display list of corpuses available for default language");

# Default file extensions
$OptL->elab(string => '.lab', Usage => "label files extension");
$OptL->eword(string => '.sts.1.words', Usage => "word files extension");
$OptL->ephon(string => '.sts.1.phones', Usage => "phoneme files extension");
$OptL->etxt(string => '.txt', Usage => "transliteration files extension");
$OptL->elist(string => '.full', Usage => "files list extension");

# Others
$OptL->dsound(string => '/cdromserv/views_pc/',Usage => "<dir>\t: root directory for speech sound files");

# Corpus and database dependent options
define_options($CORPUS,$OptL,undef);

# Remember initial settings
my $InitOptL = new OPTIONS;
$InitOptL->copy($OptL);

#----------------------------------------------------------------------
# Command processing
#----------------------------------------------------------------------
$OptL->GetOpt || die "Check your options\n";

# Check arguments

# Check the language and reload globals if necessary
if ($OptL->{groot}{string} ne $GLOBALS::GLOBALS_PATH or 
    $OptL->{lang}{string} ne $GLOBALS::DBLANG) {
  $glob->{GLOBALS_PATH} = $OptL->{groot}{string};
  $glob->{DEF_GLOBALS}  = $OptL->{groot}{string}. $OptL->{lang}{string} . '/globals.pl';
  $glob->requireglob;
}

#----------------------------------------------------------------------
# Check arguments
#----------------------------------------------------------------------
@corpuslist = keys %{$GLOBALS::CORPUS_H};

# Should we list available corpuses ?
if ($OptL->{list}{Do}) {
    # List of available corpuses
    warn "Available corpus for language $OptL->{lang}{string}: @corpuslist\n";
    exit 0;
}
# Do we have any corpus to treat ?
if (!($CORPUS=$ARGV[0])) {
    warn "Which corpus should be treated ?\n";
    $CORPUS = $corpuslist[0];
    define_options($CORPUS,$OptL,$InitOptL);
    $OptL->Usage('corpus','');
}
else {
  if (exists($GLOBALS::CORPUS_H->{$CORPUS})) {
    define_options($CORPUS,$OptL,$InitOptL);
    warn "converting labels for $CORPUS and language: $OptL->{lang}{string}\n";
    $OptL->{ldir}{string} = $GLOBALS::CORPUS_H->{$CORPUS}{list_dir};
  } else { die "corpus $CORPUS unknown for language $OptL->{lang}{string}\n";}
}


# Do we have at least one command ?

my $com_count = 0;
foreach (keys %{$OptL}) {
    $com_count += ($OptL->{$_}{Do} and $_ ne 'd2u') ;
}

if (!$com_count) {
    warn "Which command should be done ?\n";
    $OptL->Usage('corpus','');
}

#------------------------------------------------------------------------------------
# Main
#------------------------------------------------------------------------------------

# Generate list of SAM extensions for the orthographic  files of the choosen language
if (!defined($GLOBALS::CORPUS_H->{$CORPUS}{ext_list}) ){
    $GLOBALS::CORPUS_H->{$CORPUS}{ext_list} = GenSAMExtList($OptL->{lang}{string},'Orthographic');
}

# Extract some infos
my @lists =  @{$GLOBALS::CORPUS_H->{$CORPUS}{db_dirs}};
if ($OptL->{lname}{string} ne '') {@lists = ($OptL->{lname}{string});}
warn 'search paths: ', join(' ',@lists), "\n";
my $list_dir =  $OptL->{ldir}{string};
warn "list dir: $list_dir\n";
my $ext_list = $GLOBALS::CORPUS_H->{$CORPUS}{ext_list};
warn 'ext. list: ', join(" ",@{$ext_list}), "\n";

# Test the existence of list directory
if (-d $OptL->{ldir}{string}) {
    warn "convert orthographic files from lists in $OptL->{ldir}{string}\n";
    
    # Perform the treatment for each list
    foreach ( @lists ) {
	if (-d "$OptL->{dsound}{string}$_") {
 	  warn "Process list file for: $OptL->{dsound}{string}$_\n";
	  ProcessList($OptL,$_,$ext_list,$CORPUS);
	} else {
	  warn "Orthographic directory $OptL->{dsound}{string}$_ not existent\n";
       }
    }
} else {die "Error: List directory $OptL->{ldir}{string} not existent !!\n";}

__END__


=head1 COMMANDS

The following commands are defined:

=over 10

=item -d2u	

 convert from dos to unix for the other commands

=item -lab	

 create labels

=item -list	

 display list of corpuses available for default language

=item -phon	

 create phones

=item -txt	

 create transliteration

=item -word	

 create words

=back

=head1 OPTIONS

The following options are defined:

=over 14

=item -dbdir I<dir>	

 speech database directory 
 (default: /speech/db/)

=item -dsound I<dir>	

 root directory for speech sound files 
 (default: /cdromserv/views_pc/)

=item -elab I<ext>	

 label files extension 
 (default: .lab)

=item -elist I<ext>	

 files list extension 
 (default: .full)

=item -ephon I<ext>	

 phoneme files extension 
 (default: .sts.1.phones)

=item -etxt I<ext>	

 transliteration files extension 
 (default: .txt)

=item -eword I<ext>	

 word files extension 
 (default: .sts.1.words)

=item -groot I<dir>	

 global path  
 (default: /home/giron1/db/)

=item -lang I<Iso3>	

 language three letters code from Iso 3166/2T  
 (default: deu)

=item -ldir I<dir>	

 lists root directory of the defined corpus

=item -lname I<name>	

 name of list  
 (default: )

=back
